/****************************************************************************
*
*					Functions of class DataSet
*
****************************************************************************/

#include <string>
#include <iostream>
#include <fstream>
#include <vector>
#include <stdlib.h>

#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/tokenizer.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/foreach.hpp>
#include <boost/algorithm/string.hpp>

#include "DataSet.h"


/****************************************************************************
*					DataSet::DataSet
*
*	Description:	Standard constructor with initialization of internal variables
*	Parameters:		-
*	Returns:		-
*
****************************************************************************/
DataSet::DataSet (void) {
	this->clear();
}


/****************************************************************************
*					DataSet::~DataSet
*
*	Description:	Standard destructor with removing all dynamic internal variables
*	Parameters:		-
*	Returns:		-
*
****************************************************************************/
DataSet::~DataSet (void) {
	
}


/****************************************************************************
*					DataSet::readDataSet
*
*	Description:	Reads dataSet from file that is indicated in the config-file
*	Parameters:		environment - set of parameteres including where to find data set
*	Returns:		0  - if data set was successfully read
*					-1 - otherwise
*
****************************************************************************/
int DataSet::readDataSet (Environment& environment) {
	std::string dataSetDir, dataSetName;
	if ((environment.getStringParamValue(dataSetDir, "DataSetDir") != 0) || (environment.getStringParamValue(dataSetName, "DataSet"))) {
		std::cout << "\n" << "Config-file doesn't contain information about data set." << "\n";
		return -1;
	}
	return readDataSet(dataSetDir + "/" + dataSetName);
}


/****************************************************************************
*					DataSet::readDataSet
*
*	Description:	Reads data from the specified file
*	Parameters:		fileName - address of the data file
*	Returns:		0  - if data set was successfully read
*					-1 - otherwise
*
****************************************************************************/
int DataSet::readDataSet (const std::string& fileName) {
	 // Create a file reference
	 boost::filesystem::ifstream file( fileName );
	 
	 // Check if the specified file exists
	 if ( !boost::filesystem::exists( fileName ) ) {
		 std::cout << "\n" << "Specified data set file: '" << fileName << "' doesn't exists.\n";
		 return -1;
	 }

	 // Removes all previous values
	 this->clear();
	 
	 boost::char_separator<char> sep(" ;:\t,.");
 
	 int j;
	 std::string str;
	 std::vector<unsigned int> curVec;
	 unsigned int minNum, maxNum;
	 minNum = (unsigned int) -1;
	 maxNum = 0;
	 while (!file.eof())  {
		 getline(file, str);
		 if (str.empty()) 
			 continue;
		 // break string into tokens
		 token_separator tokens(str, sep);
		 // saving values
		 j = 0;
		 curVec.clear();
		 for (token_separator::iterator tok_iter = tokens.begin();  tok_iter != tokens.end();  ++tok_iter) {
			 curVec.push_back((unsigned int) atoi(tok_iter->c_str()));
		 }
		 data.push_back(curVec);
		 // some statistics
		 if (curVec.size() < minNum)
			 minNum = curVec.size();
		 if (curVec.size() > maxNum)
			 maxNum = curVec.size();
	 }

	 // Analyzing number of elements in each row:
	 if (minNum != maxNum) {
		 std::cout << "\nNumber of elements is different in several rows in the data set.\n";
	 }

	 // Setting number of samples statistics:
	 numOfSamples = data.size();

	 // Regenerate maxValues vector
	 getMaxValues();

	 // Regenerate data per feature representation
	 fillDataPerFeature();
	 
	 // Close and exit
	 file.close();
	 return 0;
 }


/****************************************************************************
*					DataSet::clear
*
*	Description:	Generate per individual feature presentation from original 
*					data set
*	Parameters:		-
*	Returns:		0
*
****************************************************************************/
int DataSet::fillDataPerFeature (void) {
	unsigned int i, j, k, l, m, len;
	// number of elements in bitwise representation of data set for a particular individual feature
	len = (data.size() + 31) / 32;
	// creating an element in dataPerFeature for each variable
	dataPerFeature.resize(maxValues.size());
	for (i = 0; i < maxValues.size(); i++) {
		// creating an element for each value of a shosen variable
		dataPerFeature[i].resize(maxValues[i] + 1);
		for (j = 0; j <= maxValues[i]; j++) {
			// creating a needed number of variables (a set of them) to represent data set in a bitwise form for a specified variable and its value
			dataPerFeature[i][j].resize (len, 0);
			if (len < 1)
				break;
			l = 0;
			m = 0;
			//int testSum = 0; // Testing correctness
			for (k = 0; k < data.size(); k++) {
				dataPerFeature[i][j][l] <<= 1;
				if (data[k][i] == j) {
					dataPerFeature[i][j][l] |= 1;
					//testSum++;  // Testing correctness
				}
				m++;
				if (m == 32) {
					m = 0;
					l++;
				}
			}
			if (m != 0)
				dataPerFeature[i][j][l] <<= (32-m);

			// Testing correctness:
			//std::cout << "\t" << i << ", " << j << ") " << testSum << " vs " << bitSum(dataPerFeature[i][j]) << " with diff = " << testSum - bitSum(dataPerFeature[i][j]) << "\n";
		}
	}
	return 0;
}


/****************************************************************************
*					DataSet::clear
*
*	Description:	Removes all variables in object internal variables
*	Parameters:		-
*	Returns:		-
*
****************************************************************************/
void DataSet::clear (void) {
	data.clear();
	dataPerFeature.clear();
	maxValues.clear();
	numOfSamples = 0;
}


/****************************************************************************
*					DataSet::getMaxValues
*
*	Description:	Sets the maximum value of each variable
*	Parameters:		-
*	Returns:		0
*
****************************************************************************/
inline int DataSet::getMaxValues(void) {
	unsigned int i,j;
	maxValues.clear();
	for (i = 0; i < data.size(); i++) {
		if (maxValues.size() < data[i].size())
			maxValues.resize(data[i].size(), 0);
		for (j = 0; j < data[i].size(); j++) {
			if (maxValues[j] < data[i][j])
				maxValues[j] = data[i][j];
		}
	}
	return 0;
}


/****************************************************************************
*					DataSet::calculateEmpiricalProb
*
*	Description:	Return empirical prbability of a given feature (defined as 
*					a description of the simple feature) in the data set
*	Parameters:		featureDescr - set of pairs (varID, varValue)
*					featureDescrSize - number of filled elements in the vector featureDescr
*	Returns:		float - empitical probability of this feature in the data set
*
****************************************************************************/
float DataSet::calculateEmpiricalProb (const std::vector <std::pair<unsigned int, unsigned int> >& featureDescr, const unsigned int& featureDescrSize) const {
	// if descriprion is empty - then there is no feature
	if (featureDescrSize == 0)
		return 0.0;
	// constructing data representation for the given feature
	std::vector <unsigned __int32> vecRep;
	vecRep = dataPerFeature[featureDescr[0].first][featureDescr[0].second];
	unsigned int i, k;
	for (i = 1; i < featureDescrSize; i++) {
		for (k = 0; k < vecRep.size(); k++)
			vecRep[k] &= dataPerFeature[featureDescr[i].first][featureDescr[i].second][k];
	}
	// returning empirical probability of the feature in the data set
	return ( ((float) (DataSet::bitSum(vecRep))) / ((float) numOfSamples) );
}


/****************************************************************************
*					DataSet::calculateEmpiricalProb
*
*	Description:	Returns empirical prbability of the given feature
*	Parameters:		feature - a feature for which to calculate empirical probability
*	Returns:		float - empitical probability of this feature in the given
*						domain in the data set
*
****************************************************************************/
float DataSet::calculateEmpiricalProb (const Feature& feature, const std::vector<unsigned int>& varIDs) const {
	// if domain is empty - then there is no feature
	if (varIDs.size() == 0)
		return 0.0;
	// getting feature desired values of the domain variables
	std::vector<unsigned int> vals;
	feature.getDesiredValues (vals);
	unsigned int numOfVals;
	numOfVals = vals.size();
	if (numOfVals == 0) {
		// this feature works not in a simple way - need to run it in every sample in the data set
		float count = 0.0;
		for (unsigned int i = 0; i < numOfSamples; i++) {
			count += feature.calculate(varIDs, data[i]);
		}
		return (count / ((float) numOfSamples));
	} else {
		// feature returned prefered values of the domain features - calculating empirical probability of these values
		if (numOfVals != varIDs.size()) {
			std::cout << "Error in DataSet::calculateEmpiricalProb: different size of domain and number of variables required by the feature.\n";
			return 0.0;
		}
		// constructing data representation for the given feature
		std::vector <unsigned __int32> vecRep;
		vecRep = dataPerFeature[varIDs[0]][vals[0]];
		unsigned int k;
		for (unsigned int i = 1; i < numOfVals; i++) {
			for (k = 0; k < vecRep.size(); k++)
				vecRep[k] &= dataPerFeature[varIDs[i]][vals[i]][k];
		}
		// returning empirical probability of the feature in the data set
		return ( ((float) (DataSet::bitSum(vecRep))) / ((float) numOfSamples) );
	}
	// should not get here because there are returns in the both branches of if
	return 0.0;
}


/****************************************************************************
*					DataSet::divideIntoTwo
*
*	Description:	Moves a given proportion of this data set samples to the new data set
*	Parameters:		proportion - a proportion of samples to move to the new data set
*	Returns:		0
*
****************************************************************************/
int DataSet::divideIntoTwo (DataSet& newDataSet, const double& proportion) {
	unsigned int numMove, numLeft, curLeave, sel;
	numMove = (unsigned int) (proportion * ( (double) numOfSamples));
	numLeft = numOfSamples - numMove;
	std::vector <bool> leave;
	leave.resize(numOfSamples, true);
	// deciding what to leave in the original data set - what to take into a new one
	curLeave = numOfSamples;
	for (unsigned int i = 0; i < numMove; i++) {
		sel = rand() % curLeave;
		while (!leave[sel])
			sel++;
		leave[sel] = false;
		curLeave--;
	}
	// moving data 
	unsigned int p1 = 0, p2 = 0;
	newDataSet.data.resize(numMove);
	for (unsigned int i = 0; i < numOfSamples; i++) {
		if (leave[i]) {
			// leaving sample in the original data set
			data[p1] = data[i];
			p1++;
		} else {
			// moving sample into the new data set
			newDataSet.data[p2] = data[i];
			p2++;
		}
	}
	// Checking correctness:
	if ((p1 != numLeft) || (p2 != numMove))
		std::cout << "ERROR in DataSet::divideIntoTwo: wrong division of the data set into parts.\n";
	
	// changing the size of the data set
	data.resize(numLeft);
	newDataSet.data.resize(numMove);

	// Setting number of samples statistics:
	numOfSamples = numLeft;
	newDataSet.numOfSamples = numMove;
	
	// Setting maxValues vector
	newDataSet.maxValues = maxValues;
	
	// Regenerate data per feature representation
	fillDataPerFeature();
	newDataSet.fillDataPerFeature();

	return 0;
}


/****************************************************************************
*					DataSet::bitSum
*
*	Description:	Funciton returns a sum of 1's in the bit representation 
*					of a given number u
*	Parameters:		u - input number to calculate ones in its bit representation
*	Returns:		int - number of ones
*
****************************************************************************/
/*unsigned int DataSet::bitSum(unsigned __int32& u) {
	unsigned __int32 uCount;
	uCount = u - ((u >> 1) & 033333333333) - ((u >> 2) & 011111111111);
	return (unsigned int) (((uCount + (uCount >> 3)) & 030707070707) % 63);
}*/


/****************************************************************************
*					DataSet::bitSum
*
*	Description:	Funciton returns a sum of 1's in the bit representation 
*					of numbers in a given vector
*	Parameters:		u - input vector of numbers to calculate ones in their bit representation
*	Returns:		int - number of ones
*
****************************************************************************/
/*unsigned int DataSet::bitSum(std::vector<unsigned __int32>& v) {
	unsigned int res = 0;
	for (unsigned int i = 0; i < v.size(); i++)
		res += bitSum(v[i]);
	return res;
}*/
